package umich.max.geolocation.clean;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;

import max.nlp.dal.annotations.DBpediaAnnotation;
import max.nlp.dal.blog.blogger.BlogAuthorProfile;
import max.nlp.dal.blog.blogger.BloggerBlog;
import max.nlp.dal.blog.blogger.BloggerDB;
import max.nlp.dal.blog.blogger.BloggerPost;
import max.nlp.dal.blog.blogger.ParsedLocation;
import max.nlp.wrappers.dbpedia.DBpediaSpotlight;
import max.nlp.wrappers.stanford.StanfordNER;

import org.jsoup.Jsoup;

import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCursor;

public class BlogCleaner {

	public static void main(String[] args) {
		// cleanAllPosts();
		// // nerAllPosts();
		 disambiguateLocations();
		// runThruDBpedia();
		// detectDuplicates();
		// detectSmallPosts();
//		test();
//		print();
	}

	public static void test() {
		BloggerDB db = BloggerDB.getInstance();
		DBCursor itr = db.getBlogIterator();
		BloggerBlog blog = db
				.findBlogbyURL("http://flychallenger.blogspot.com/");
		for (BloggerPost post : blog.getPosts()) {
			System.out.println(post.getCleanContent());
			System.out.println(post.getLocations().size());
			for (String loc : post.getLocations())
				System.out.print(loc + "\t");
			System.out.println();
		}
	}

	public static  void print() {
		BloggerDB db = BloggerDB.getInstance();
		DBCursor itr = db.getBlogIterator();
		while (itr.hasNext()) {
			BloggerBlog blog = new BloggerBlog(itr.next());
			for (BloggerPost post : blog.getPosts()) {
				if (post.getLocations().size() > 0)
					System.out.println(blog.getUrl());
			}
		}
	}

	public static void cleanAllPosts() {
		BloggerDB db = BloggerDB.getInstance();
		DBCursor itr = db.getBlogIterator();
		while (itr.hasNext()) {
			BloggerBlog blog = new BloggerBlog(itr.next());
			if (!hasBlogBeenCleaned(blog)) {
				List<BloggerPost> posts = blog.getPosts();
				if (posts != null) {
					BasicDBList cleanedPosts = cleanPosts(posts);

					BasicDBObject updateQuery = new BasicDBObject();
					updateQuery.put("blogID", blog.getblogID());

					BasicDBObject updateCommand = new BasicDBObject();
					BasicDBObject newValues = new BasicDBObject("posts",
							cleanedPosts);
					updateCommand.put("$set", newValues);

					db.updatePosts(updateQuery, updateCommand);

				}
			}
		}
	}

	public static void detectDuplicates() {
		BloggerDB db = BloggerDB.getInstance();
		HashSet<String> urls = new HashSet<String>();
		DBCursor itr = db.getBlogIterator();
		int dupes = 0;
		while (itr.hasNext()) {
			BloggerBlog blog = new BloggerBlog(itr.next());
			String url = blog.getUrl();
			if (urls.contains(url)) {
				dupes++;
				System.out.println(url);
			} else
				urls.add(url);
		}
		System.out.println(dupes);
	}

	public static void detectSmallPosts() {
		final int smallSize = 3;
		BloggerDB db = BloggerDB.getInstance();
		HashSet<String> urls = new HashSet<String>();
		DBCursor itr = db.getBlogIterator();
		int total = 0;
		int big = 0;
		while (itr.hasNext()) {
			BloggerBlog blog = new BloggerBlog(itr.next());
			int numPosts = blog.getPosts().size();
			if (numPosts > smallSize)
				big++;
			total++;
		}
		System.out.println(big);
		System.out.println(total);
	}

	public static boolean hasBlogBeenCleaned(BloggerBlog b) {
		List<BloggerPost> posts = b.getPosts();
		if (posts.isEmpty())
			return true;
		else {
			for (BloggerPost p : posts) {
				if (p.getCleanContent() == null)
					return false;
			}
			return true;
		}

	}

	public static boolean hasBlogBeenNERd(BloggerBlog b) {
		List<BloggerPost> posts = b.getPosts();
		if (posts.isEmpty())
			return true;
		else {
			for (BloggerPost p : posts) {
				if (!p.getLocations().isEmpty()
						|| !p.getOrganization().isEmpty()
						|| !p.getPeople().isEmpty())
					return true;
			}
			return false;
		}

	}

	public static void nerAllPosts() {
		BloggerDB db = BloggerDB.getInstance();
		StanfordNER ner = StanfordNER.getInstance();

		DBCursor itr = db.getBlogIterator();
		while (itr.hasNext()) {
			BloggerBlog blog = new BloggerBlog(itr.next());
			List<BloggerPost> posts = blog.getPosts();
			if (posts != null && !hasBlogBeenNERd(blog)) {
				System.out.println(blog.getUrl());
				for (BloggerPost p : posts) {
					String text = p.getCleanContent();
					HashMap<String, List<String>> ents = ner
							.extractEntities(text);

					for (Entry<String, List<String>> e : ents.entrySet()) {
						String entityType = e.getKey();
						List<String> vals = e.getValue();
						if (entityType.equals("LOCATION") && !vals.isEmpty()) {
							db.updatePost(p, "posts.$.locations", vals, blog);

						} else if (entityType.equals("ORGANIZATION")
								&& !vals.isEmpty()) {
							db.updatePost(p, "posts.$.organization", vals, blog);

						} else if (entityType.equals("PERSON")
								&& !vals.isEmpty()) {
							db.updatePost(p, "posts.$.PEOPLE", vals, blog);

						}
					}

				}
			}
		}
	}

	public static BasicDBList cleanPosts(List<BloggerPost> oldPosts) {
		BasicDBList newPosts = new BasicDBList();
		for (BloggerPost oldPost : oldPosts) {
			String uncleanContent = oldPost.getContent();
			if (uncleanContent != null) {
				String cleanContent = Jsoup.parse(uncleanContent).text();
				oldPost.setCleanContent(cleanContent);
				newPosts.add(oldPost);
			}
		}
		return newPosts;
	}

	public static void disambiguateLocations() {
		try {
			PrintWriter out = new PrintWriter(new FileWriter(new File(
					"/home/max/temp-with.txt")));
			BloggerDB db = BloggerDB.getInstance();
			DBCursor itr = db.getBlogIterator();
			while (itr.hasNext()) {
				BloggerBlog blog = new BloggerBlog(itr.next());
				BlogAuthorProfile profile = db.findProfileForBlog(blog);
				ParsedLocation pLoc = profile.getParsedLocation();

				for (BloggerPost post : blog.getPosts()) {
					List<String> locs = post.getLocations();
					if (!locs.isEmpty()) {
						out.println("**********");
						if (pLoc != null
								&& (pLoc.getCity() != null || pLoc.getState() != null))
							out.println(pLoc);
						for (String loc : locs) {
							out.println(loc);
						}
						out.println(post.getCleanContent());
						out.println("***********\n");
						out.flush();
					}
				}
			}
			out.close();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	


	public static void runThruDBpedia() {

		BloggerDB db = BloggerDB.getInstance();

		DBpediaSpotlight dbpedia = new DBpediaSpotlight();
		DBCursor itr = db.getBlogIterator();
		int count = 0;
		while (itr.hasNext()) {
			System.out.println(count++);
			BloggerBlog blog = new BloggerBlog(itr.next());
			if (hasBlogBeenCleaned(blog)) {
				List<BloggerPost> posts = blog.getPosts();
				if (posts != null) {
					for (BloggerPost post : posts) {
						String text = post.getCleanContent();
						List<DBpediaAnnotation> annotations = dbpedia
								.sendPost(text);
						if (!annotations.isEmpty()) {
							db.updatePost(post, "posts.$.dbpediaAnnotation",
									annotations, blog);
						}

					}

				}
			}
		}
	}
}
